{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ning\\AppData\\Local\\Temp\\ipykernel_12520\\2634863375.py:83: DeprecationWarning: The 'text' argument to find()-type methods is deprecated. Use 'string' instead.\n",
      "  deadline_tag = each_notice.bs.find(text=re.compile(r'截止时间：'))\n",
      "C:\\Users\\Ning\\AppData\\Local\\Temp\\ipykernel_12520\\2634863375.py:90: DeprecationWarning: The 'text' argument to find()-type methods is deprecated. Use 'string' instead.\n",
      "  contact_tag = each_notice.bs.find(text=re.compile(r'联系人：'))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Added project: 数字法学学科建设学术论坛通知\n",
      "Added project: 申报辽宁省高等学校新型智库联盟2024年度智库专项课题\n",
      "Added project: 决策咨询类科研成果认定\n",
      "Added project: 公布鼎创实验班第一期录取名单\n",
      "Added project: 申报国家发改委区域协调发展司2024年度研究课题\n",
      "Added project: 2023年度著作类科研成果评级\n",
      "Added project: “经济计量分析与预测国际学术会议（2024年）”征文通知\n",
      "Added project: 开展2024年度大连市科学技术奖提名\n",
      "Added project: 申报2024年全国文化艺术职业教育和旅游职业教育提质培优行动计划项目\n",
      "Added project: 申报国家发改委经济运行调节局2024年度研究课题\n",
      "Added project: 2024年上半年毕业研究生提交学位论文\n",
      "Added project: 2024年度校级教学改革研究项目立项申报\n",
      "Added project: 申报民政部规划财务司“十五五”研究课题\n",
      "Added project: 申报大连市社科院2024年度课题\n",
      "Added project: 在全校党外知识分子中开展“我为发展献良策 凝心聚力促振兴”活动\n",
      "Added project: 申报2024年度全国教育科学规划项目\n",
      "Added project: 申报2024年度国家社科基金教育学重大项目\n",
      "Added project: 申报文化和旅游部办公厅2024年度国家文化和旅游科技创新研发项目\n",
      "Added project: 申报2024年度辽宁省科协科技创新智库项目\n",
      "Added project: 申报农业农村部农村经济研究中心2024年度“三农”政策与理论研究课题\n",
      "Added project: 申报2024年辽宁省优秀自然科学学术著作出版资助\n",
      "Added project: 申报国家发改委国际合作司2024年度研究课题\n",
      "Added project: 严明“五一”“端午”节日期间纪律要求\n",
      "Added project: 鼎创实验班第一期招生简章\n",
      "Added project: 征集2024年度法学课题研究选题\n",
      "Added project: 申报第七届刘诗白经济学奖\n",
      "Added project: 申报国家发改委国民经济综合司2024年度研究课题\n",
      "Added project: 申报2024年度国家社会科学基金重大项目\n",
      "Added project: 申报2024年度全国统计科学研究项目\n",
      "Added project: 申报2024年国家社会科学基金年度项目\n",
      "Added project: 捐出书籍，共享书香——东北财经大学图书馆图书捐赠倡议\n",
      "Added project: 申报国家民委民族研究项目2024年度课题\n",
      "Added project: 东北财经大学师生报名“2024年亚洲计量经济学与统计学暑期学校”的补充说明\n",
      "Added project: 申报2024年度最高人民检察院检察理论研究课题\n",
      "Added project: 2024年亚洲计量经济学与统计学暑期学校报名通知\n",
      "Added project: 征集教育部学位与研究生教育发展中心工商管理“精品案例课堂”项目\n",
      "Added project: 参加首届国际商务专业学位研究生国际贸易融资竞赛\n",
      "Added project: 辽宁省2024年度选调应届优秀大学毕业生工作公告\n",
      "Added project: 图书馆2023年度个人阅读报告发布\n",
      "Added project: 东北财经大学公开招聘10人，现开始报名\n",
      "Added project: 东北财经大学校级提升社会服务能力建设项目、省级以上科研平台支持专项结项评审结果\n",
      "Added project: 公布2023年度校级教学改革研究项目（含虚拟教研室类）结题验收结果\n",
      "Added project: 2024年新入职教师岗前培训\n",
      "Added project: 发放东北财经大学2023年教职工住房货币补贴\n",
      "Added project: 确定2023年校级廉洁教育工作创新项目立项课题\n",
      "Added project: 公布2023年度校级教学改革研究项目（虚拟教研室建设）立项名单\n",
      "Added project: 公布2023年度校级教学改革研究项目立项名单\n",
      "Added project: 开展大连市科协重要学术交流活动征集\n",
      "Added project: 开展2023年度东北财经大学党建工作质量创优和精品创建\n",
      "数据已保存到 20240523.csv\n",
      "数据已保存到数据库 notifications.db 中。\n"
     ]
    }
   ],
   "source": [
    "import requests as rq\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import pandas as pd\n",
    "import sqlite3\n",
    "from datetime import datetime\n",
    "\n",
    "url_generator = (fr'https://www.dufe.edu.cn/r_6_{i}.html' for i in range(1, 4))\n",
    "\n",
    "class Notice:\n",
    "    def __init__(self, url):\n",
    "        self.url = r'https://www.dufe.edu.cn/' + url\n",
    "        self.page = rq.get(self.url).text\n",
    "        self.bs = BeautifulSoup(self.page, 'html.parser')\n",
    "        self.title = ''\n",
    "        for each_element in self.bs.findAll('div', attrs={\n",
    "            'class': 'crumbs underline col-lg-8 col-md-8 col-sm-8 col-xs-12'}):\n",
    "            title_list = re.findall(r'\\n-\\s(.*?)\\n', each_element.text)\n",
    "            if title_list:\n",
    "                self.title = title_list[0]\n",
    "        \n",
    "        self.date = ''\n",
    "        for each_element in self.bs.findAll(\"div\", attrs={\"class\": \"color-6 h5 xwnr_date_mb\"}):\n",
    "            self.date = each_element.text\n",
    "        \n",
    "        self.content = ''\n",
    "        for each_element in self.bs.findAll('p', attrs={'style': 'text-align: justify;'}):\n",
    "            self.content += each_element.text\n",
    "        if self.content == '':\n",
    "            for each_element in self.bs.findAll('div', attrs={'class': 'h5 xwnr_content ue_content ue_content_side'}):\n",
    "                self.content += each_element.text\n",
    "        \n",
    "        self.type = '其他'\n",
    "        if '课题' in self.title:\n",
    "            self.type = '课题申报'\n",
    "        elif '结题' in self.title:\n",
    "            self.type = '项目结题'\n",
    "        elif '成果认定' in self.title:\n",
    "            self.type = '成果认定'\n",
    "        elif '成果评级' in self.title:\n",
    "            self.type = '成果评级'\n",
    "\n",
    "def save_to_csv(data):\n",
    "    today = datetime.today().strftime('%Y%m%d')\n",
    "    csv_filename = f'{today}.csv'\n",
    "    df = pd.DataFrame(data, columns=['项目名称', '项目类型', '发布日期', '截止日期', '联系人', '网页URL'])\n",
    "    df.to_csv(csv_filename, index=False, encoding='utf-8-sig')\n",
    "    print(f'数据已保存到 {csv_filename}')\n",
    "\n",
    "def save_to_db(data):\n",
    "    conn = sqlite3.connect('notifications.db')\n",
    "    df = pd.DataFrame(data, columns=['项目名称', '项目类型', '发布日期', '截止日期', '联系人', '网页URL'])\n",
    "    df.to_sql('notifications', conn, if_exists='append', index=False)\n",
    "    conn.close()\n",
    "    print('数据已保存到数据库 notifications.db 中。')\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    data = []\n",
    "\n",
    "    for each_source_url in url_generator:\n",
    "        source_html = rq.get(each_source_url).text\n",
    "        mybs = BeautifulSoup(source_html, \"html.parser\")\n",
    "        all_notice = mybs.findAll(\"a\", attrs={\"class\": \"color-3\", 'target': '_blank'})\n",
    "\n",
    "        for each in all_notice:\n",
    "            each_notice = Notice(url=each['href'])\n",
    "            project_name = each_notice.title\n",
    "            project_url = each_notice.url\n",
    "\n",
    "            if any(keyword in project_name for keyword in [\"研讨会\", \"实践\"]):\n",
    "                continue\n",
    "\n",
    "            project_name = re.sub(r'关于组织|关于|工作的通知|的通知', '', project_name)\n",
    "\n",
    "            publish_date = each_notice.date\n",
    "\n",
    "            deadline_date = ''\n",
    "            deadline_tag = each_notice.bs.find(text=re.compile(r'截止时间：'))\n",
    "            if deadline_tag:\n",
    "                deadline_date_match = re.search(r'\\d{4}年\\d{1,2}月\\d{1,2}日', deadline_tag)\n",
    "                deadline_date = deadline_date_match.group() if deadline_date_match else ''\n",
    "\n",
    "            contact_person = ''\n",
    "            contact_tag = each_notice.bs.find(text=re.compile(r'联系人：'))\n",
    "            if contact_tag:\n",
    "                contact_person = contact_tag.split('：')[1].strip()\n",
    "\n",
    "            project_type = each_notice.type\n",
    "\n",
    "            data.append([project_name, project_type, publish_date, deadline_date, contact_person, project_url])\n",
    "            print(f\"Added project: {project_name}\")\n",
    "    if data:\n",
    "        save_to_csv(data)\n",
    "        save_to_db(data)\n",
    "    else:\n",
    "        print('No data to save.')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---------------------------------------------------------------------------\n",
    "AttributeError                            Traceback (most recent call last)\n",
    "Cell In[4], line 126\n",
    "    123     return body\n",
    "    125 if __name__ == \"__main__\":\n",
    "--> 126     main()\n",
    "\n",
    "Cell In[4], line 93\n",
    "     91 def main():\n",
    "     92     url = 'https://www.dufe.edu.cn/r_6.html'\n",
    "---> 93     notice_data = scrape_notice_page(url)\n",
    "     94     for notice in notice_data:\n",
    "     95         notice['title'] = clean_title(notice['title'])\n",
    "\n",
    "Cell In[4], line 17\n",
    "     15 notice_list = soup.find('ul', class_='xw-list')\n",
    "     16 results = []\n",
    "---> 17 for notice in notice_list.find_all('li'):\n",
    "     18     title_elem = notice.find('a', class_='color-3')\n",
    "     19     title = title_elem.get_text(strip=True)\n",
    "\n",
    "AttributeError: 'NoneType' object has no attribute 'find_all'"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "MyMLenvAllinOne",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
